{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d15d8294-3328-4e07-ad16-8a03e9bbfdb9",
   "metadata": {},
   "source": [
    "# How to run a cell\n",
    "\n",
    "Press `Shift` + `Return` to run a Cell.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e2a9393-7767-488e-a8bf-27c12dca35bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "\n",
    "import os, requests, time\n",
    "from dotenv import load_dotenv\n",
    "from bs4 import BeautifulSoup\n",
    "from IPython.display import Markdown, display\n",
    "from openai import OpenAI\n",
    "\n",
    "# Load environment variables in a file called .env\n",
    "load_dotenv(override=True)\n",
    "api_key = os.getenv('OPENAI_API_KEY')\n",
    "\n",
    "# Check the key\n",
    "if not api_key:\n",
    "    print(\"No API key was found\")\n",
    "else:\n",
    "    print(\"API key found and looks good so far!\")\n",
    "\n",
    "# Instantiate an OpenAI object\n",
    "openai = OpenAI()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "442fc84b-0815-4f40-99ab-d9a5da6bda91",
   "metadata": {},
   "source": [
    "# Make a test call to a Frontier model (Open AI) to get started:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a58394bf-1e45-46af-9bfd-01e24da6f49a",
   "metadata": {},
   "outputs": [],
   "source": [
    "message = \"Hello, GPT! Holla back to this space probe!\"\n",
    "response = openai.chat.completions.create(model=\"gpt-4o-mini\", messages=[{\"role\":\"user\", \"content\":message}])\n",
    "print(response.choices[0].message.content)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2aa190e5-cb31-456a-96cc-db109919cd78",
   "metadata": {},
   "source": [
    "## Summarization project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5e793b2-6775-426a-a139-4848291d0463",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some websites need proper headers when fetching them:\n",
    "headers = {\n",
    " \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36\"\n",
    "}\n",
    "\n",
    "\"\"\"\n",
    "A class to represent a Webpage\n",
    "\"\"\"\n",
    "class Website:\n",
    "\n",
    "    def __init__(self, url):\n",
    "        \"\"\"\n",
    "        Create this Website object from the given url using the BeautifulSoup library\n",
    "        \"\"\"\n",
    "        self.url = url\n",
    "        response = requests.get(url, headers=headers)\n",
    "        soup = BeautifulSoup(response.content, 'html.parser')\n",
    "        self.title = soup.title.string if soup.title else \"No title found\"\n",
    "        for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n",
    "            irrelevant.decompose()\n",
    "        self.text = soup.body.get_text(separator=\"\\n\", strip=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ef960cf-6dc2-4cda-afb3-b38be12f4c97",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Summarize website content\n",
    "website = Website(\"https://rwothoromo.wordpress.com/\")\n",
    "# print(eli.title, \"\\n\", eli.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abdb8417-c5dc-44bc-9bee-2e059d162699",
   "metadata": {},
   "outputs": [],
   "source": [
    "# A system prompt tells a model like GPT4o what task they are performing and what tone they should use\n",
    "# A user prompt is the conversation starter that they should reply to\n",
    "\n",
    "system_prompt = \"You are an assistant that analyzes the contents of a given website, \\\n",
    "and returns a brief summary, ignoring text that might be navigation-related. \\\n",
    "Respond in markdown.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0275b1b-7cfe-4f9d-abfa-7650d378da0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# A function that writes a User Prompt that asks for summaries of websites:\n",
    "\n",
    "def user_prompt_for(website):\n",
    "    user_prompt = f\"You are looking at a website titled {website.title}\"\n",
    "    user_prompt += \"\\nThe contents of this website is as follows; \\\n",
    "please provide a short summary of this website in markdown. \\\n",
    "If it includes news or announcements, then summarize these too.\\n\\n\"\n",
    "    user_prompt += website.text\n",
    "    return user_prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26448ec4-5c00-4204-baec-7df91d11ff2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(user_prompt_for(website))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f25dcd35-0cd0-4235-9f64-ac37ed9eaaa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The API from OpenAI expects to receive messages in a particular structure. Many of the other APIs share this structure:\n",
    "messages = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a snarky assistant\"}, # system message\n",
    "    {\"role\": \"user\", \"content\": \"What is 2 + 2?\"}, # user message\n",
    "]\n",
    "response = openai.chat.completions.create(model=\"gpt-4o-mini\", messages=messages)\n",
    "print(response.choices[0].message.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0134dfa4-8299-48b5-b444-f2a8c3403c88",
   "metadata": {},
   "outputs": [],
   "source": [
    "# To build useful messages for GPT-4o-mini\n",
    "\n",
    "def messages_for(website):\n",
    "    return [\n",
    "        {\"role\": \"system\", \"content\": system_prompt},\n",
    "        {\"role\": \"user\", \"content\": user_prompt_for(website)}\n",
    "    ]\n",
    "\n",
    "messages_for(website)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "905b9919-aba7-45b5-ae65-81b3d1d78e34",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call the OpenAI API.\n",
    "\n",
    "url = \"https://rwothoromo.wordpress.com/\"\n",
    "website = Website(url)\n",
    "\n",
    "def summarize(website):\n",
    "    response = openai.chat.completions.create(\n",
    "        model = \"gpt-4o-mini\",\n",
    "        messages = messages_for(website)\n",
    "    )\n",
    "    return response.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05e38d41-dfa4-4b20-9c96-c46ea75d9fb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "summarize(website)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d926d59-450e-4609-92ba-2d6f244f1342",
   "metadata": {},
   "outputs": [],
   "source": [
    "# A function to display this nicely in the Jupyter output, using markdown\n",
    "\n",
    "summary = summarize(website)\n",
    "def display_summary(summary):\n",
    "    display(Markdown(summary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3018853a-445f-41ff-9560-d925d1774b2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "display_summary(summary)\n",
    "# display_summary(summarize(Website(\"https://edwarddonner.com\")))\n",
    "# display_summary(summarize(Website(\"https://cnn.com\")))\n",
    "# display_summary(summarize(Website(\"https://anthropic.com\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a904323-acd9-4c8e-9a17-70df76184590",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Websites protected with CloudFront (and similar) or with JavaScript need a Selenium or Playwright implementation. They return 403\n",
    "\n",
    "# display_summary(summarize(Website(\"https://openai.com\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "139ad985",
   "metadata": {},
   "outputs": [],
   "source": [
    "# To generate the above summary, use selenium\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "\n",
    "class WebsiteSelenium:\n",
    "    def __init__(self, url):\n",
    "        self.url = url\n",
    "        self.title = \"No title found\"\n",
    "        self.text = \"\"\n",
    "\n",
    "        # Configure Chrome options (headless mode is recommended for server environments)\n",
    "        chrome_options = webdriver.ChromeOptions()\n",
    "        chrome_options.add_argument(\"--headless\")  # Run Chrome in headless mode (without a UI)\n",
    "        chrome_options.add_argument(\"--no-sandbox\") # Required for running as root in some environments\n",
    "        chrome_options.add_argument(\"--disable-dev-shm-usage\") # Overcomes limited resource problems\n",
    "\n",
    "        # Path to your WebDriver executable (e.g., chromedriver)\n",
    "        # Make sure to replace this with the actual path to your chromedriver\n",
    "        # You might need to download it from: https://chromedriver.chromium.org/downloads and place it in a drivers dir\n",
    "        service = Service('./drivers/chromedriver-mac-x64/chromedriver')\n",
    "\n",
    "        driver = None\n",
    "        try:\n",
    "            driver = webdriver.Chrome(service=service, options=chrome_options)\n",
    "            driver.get(url)\n",
    "\n",
    "            # Wait for the page to load and dynamic content to render\n",
    "            # You might need to adjust the wait condition based on the website\n",
    "            WebDriverWait(driver, 10).until(\n",
    "                EC.presence_of_element_located((By.TAG_NAME, \"body\"))\n",
    "            )\n",
    "            time.sleep(3) # Give more time for JavaScript to execute\n",
    "\n",
    "            # Get the page source after dynamic content has loaded\n",
    "            soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
    "\n",
    "            self.title = soup.title.string if soup.title else \"No title found\"\n",
    "            for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n",
    "                irrelevant.decompose()\n",
    "            self.text = soup.body.get_text(separator=\"\\n\", strip=True)\n",
    "\n",
    "        except Exception as e:\n",
    "            print(f\"Error accessing {url} with Selenium: {e}\")\n",
    "        finally:\n",
    "            if driver:\n",
    "                driver.quit() # Always close the browser\n",
    "\n",
    "display_summary(summarize(WebsiteSelenium(\"https://openai.com\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "130d4572",
   "metadata": {},
   "outputs": [],
   "source": [
    "import asyncio\n",
    "from playwright.async_api import async_playwright\n",
    "import nest_asyncio\n",
    "\n",
    "# Apply nest_asyncio to allow asyncio.run in Jupyter\n",
    "nest_asyncio.apply()\n",
    "\n",
    "class WebsitePlaywright:\n",
    "    def __init__(self, url):\n",
    "        self.url = url\n",
    "        self.title = \"No title found\"\n",
    "        self.text = \"\"\n",
    "        asyncio.run(self._fetch_content())\n",
    "\n",
    "    async def _fetch_content(self):\n",
    "        async with async_playwright() as p:\n",
    "            browser = None\n",
    "            try:\n",
    "                browser = await p.chromium.launch(headless=True)\n",
    "                page = await browser.new_page()\n",
    "\n",
    "                # Increase timeout for navigation and other operations\n",
    "                await page.goto(self.url, timeout=60000) # Wait up to 60 seconds for navigation\n",
    "                print(f\"Accessing {self.url} with Playwright - goto()\")\n",
    "\n",
    "                # You might need to adjust or add more specific waits\n",
    "                await page.wait_for_load_state('domcontentloaded', timeout=60000) # Wait for basic HTML\n",
    "                # await page.wait_for_load_state('networkidle', timeout=60000) # Wait for network activity to settle\n",
    "                await page.wait_for_selector('div.duration-short', timeout=60000) # instead of networkidle\n",
    "                await page.wait_for_selector('body', timeout=60000) # Wait for the body to be present\n",
    "                await asyncio.sleep(5) # Give a bit more time for final rendering\n",
    "\n",
    "                content = await page.content()\n",
    "                soup = BeautifulSoup(content, 'html.parser')\n",
    "\n",
    "                self.title = soup.title.string if soup.title else \"No title found\"\n",
    "                for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n",
    "                    irrelevant.decompose()\n",
    "                self.text = soup.body.get_text(separator=\"\\n\", strip=True)\n",
    "                print(f\"Accessed {self.url} with Playwright\")\n",
    "\n",
    "            except Exception as e:\n",
    "                print(f\"Error accessing {self.url} with Playwright: {e}\")\n",
    "            finally:\n",
    "                if browser:\n",
    "                    await browser.close()\n",
    "\n",
    "display_summary(summarize(WebsitePlaywright(\"https://openai.com/\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00743dac-0e70-45b7-879a-d7293a6f68a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1: Create your prompts\n",
    "\n",
    "system_prompt = \"You are a professional assistant. Review this conversation and provide a comprehensive summary. Also, suggest how much better the converation could have gone:\"\n",
    "user_prompt = \"\"\"\n",
    "\n",
    "Dear Email Contact,\n",
    "\n",
    "I hope this message finds you well.\n",
    "I would like to share that I have proficiency in front-end design tools, particularly Figma, react and Angular. At this stage, I am keenly interested in finding opportunities to apply these skills professionally.\n",
    "\n",
    "If you are aware of any companies, projects, or platforms seeking enterprise in front-end design, I would be grateful for any advice or recommendations you might kindly provide.\n",
    "\n",
    "Thank you very much for your time and consideration.\n",
    "\n",
    "Hello Job Seeker,\n",
    "\n",
    "I hope you are doing well.\n",
    "\n",
    "The last role (3 months gig) I saw was looking for a junior PHP Developer. Does your CV include that?\n",
    "\n",
    "Hello Email Contact,\n",
    "Thank you for your feedback.\n",
    "Yes my CV has PHP as one of my skill set. Can I share it with you?\n",
    "\n",
    "Email Contact: They said \"It's late. Interviews were on Monday\"\n",
    "\n",
    "Hello Email Contact\n",
    "\n",
    "Thanks for the update. When you hear of any opportunity please let me know.\n",
    "\n",
    "Email Contact: For now, check out https://refactory.academy/courses/refactory-apprenticeship/\n",
    "\"\"\"\n",
    "\n",
    "# Step 2: Make the messages list\n",
    "\n",
    "messages = [\n",
    "    {\"role\": \"system\", \"content\": system_prompt},\n",
    "    {\"role\": \"user\", \"content\": user_prompt},\n",
    "]\n",
    "\n",
    "# Step 3: Call OpenAI\n",
    "\n",
    "response = openai.chat.completions.create(\n",
    "    model = \"gpt-4o-mini\",\n",
    "    messages = messages\n",
    ")\n",
    "\n",
    "# Step 4: print the result\n",
    "\n",
    "print(response.choices[0].message.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b583226-9b13-4990-863a-86517a5ccfec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# To perform summaries using a model running locally\n",
    "import ollama\n",
    "\n",
    "# OLLAMA_API = \"http://localhost:11434/api/chat\"\n",
    "# HEADERS = {\"Content-Type\": \"application/json\"}\n",
    "MODEL = \"llama3.2\"\n",
    "\n",
    "def summarize_with_local_model(url):\n",
    "    website = Website(url)\n",
    "    messages = messages_for(website)\n",
    "    response = ollama.chat(\n",
    "        model=MODEL,\n",
    "        messages=messages,\n",
    "        stream=False # just get the results, don't stream them\n",
    "        )\n",
    "    return response['message']['content']\n",
    "\n",
    "display(Markdown(summarize_with_local_model(\"https://rwothoromo.wordpress.com/\")))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
